{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (C) 2016 - 2019 Pinard Liu(liujianping-ok@163.com)\n",
    "\n",
    "https://www.cnblogs.com/pinard\n",
    "\n",
    "Permission given to modify the code as long as you keep this declaration at the top\n",
    "\n",
    "中文文本挖掘预处理流程总结 https://www.cnblogs.com/pinard/p/6744056.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "import jieba\n",
    "\n",
    "with open('./nlp_test0.txt') as f:\n",
    "    document = f.read()\n",
    "    \n",
    "    document_decode = document.decode('GBK')\n",
    "    document_cut = jieba.cut(document_decode)\n",
    "    #print  ' '.join(jieba_cut)  //如果打印结果，则分词效果消失，后面的result无法显示\n",
    "    result = ' '.join(document_cut)\n",
    "    result = result.encode('utf-8')\n",
    "    with open('./nlp_test1.txt', 'w') as f2:\n",
    "        f2.write(result)\n",
    "f.close()\n",
    "f2.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jieba.suggest_freq('沙瑞金', True)\n",
    "jieba.suggest_freq('易学习', True)\n",
    "jieba.suggest_freq('王大路', True)\n",
    "jieba.suggest_freq('京州', True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./nlp_test0.txt') as f:\n",
    "    document = f.read()\n",
    "    \n",
    "    document_decode = document.decode('GBK')\n",
    "    document_cut = jieba.cut(document_decode)\n",
    "    #print  ' '.join(jieba_cut)\n",
    "    result = ' '.join(document_cut)\n",
    "    result = result.encode('utf-8')\n",
    "    with open('./nlp_test1.txt', 'w') as f2:\n",
    "        f2.write(result)\n",
    "f.close()\n",
    "f2.close()     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#从文件导入停用词表\n",
    "stpwrdpath = \"stop_words.txt\"\n",
    "stpwrd_dic = open(stpwrdpath, 'rb')\n",
    "stpwrd_content = stpwrd_dic.read()\n",
    "#将停用词表转换为list  \n",
    "stpwrdlst = stpwrd_content.splitlines()\n",
    "stpwrd_dic.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "沙瑞金 赞叹 易学习 的 胸怀 ， 是 金山 的 百姓 有福 ， 可是 这件 事对 李达康 的 触动 很大 。 易学习 又 回忆起 他们 三人 分开 的 前一晚 ， 大家 一起 喝酒 话别 ， 易学习 被 降职 到 道口 县当 县长 ， 王大路 下海经商 ， 李达康 连连 赔礼道歉 ， 觉得 对不起 大家 ， 他 最 对不起 的 是 王大路 ， 就 和 易学习 一起 给 王大路 凑 了 5 万块 钱 ， 王大路 自己 东挪西撮 了 5 万块 ， 开始 下海经商 。 没想到 后来 王大路 竟然 做 得 风生水 起 。 沙瑞金 觉得 他们 三人 ， 在 困难 时期 还 能 以沫 相助 ， 很 不 容易 。\n"
     ]
    }
   ],
   "source": [
    "with open('./nlp_test1.txt') as f3:\n",
    "    res1 = f3.read()\n",
    "print res1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./nlp_test2.txt') as f:\n",
    "    document2 = f.read()\n",
    "    \n",
    "    document2_decode = document2.decode('GBK')\n",
    "    document2_cut = jieba.cut(document2_decode)\n",
    "    #print  ' '.join(jieba_cut)\n",
    "    result = ' '.join(document2_cut)\n",
    "    result = result.encode('utf-8')\n",
    "    with open('./nlp_test3.txt', 'w') as f2:\n",
    "        f2.write(result)\n",
    "f.close()\n",
    "f2.close()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "沙瑞金 向 毛娅 打听 他们 家 在 京州 的 别墅 ， 毛娅 笑 着 说 ， 王大路 事业有成 之后 ， 要 给 欧阳 菁 和 她 公司 的 股权 ， 她们 没有 要 ， 王大路 就 在 京州 帝豪园 买 了 三套 别墅 ， 可是 李达康 和 易学习 都 不要 ， 这些 房子 都 在 王大路 的 名下 ， 欧阳 菁 好像 去 住 过 ， 毛娅 不想 去 ， 她 觉得 房子 太大 很 浪费 ， 自己 家住 得 就 很 踏实 。\n"
     ]
    }
   ],
   "source": [
    "with open('./nlp_test3.txt') as f4:\n",
    "    res2 = f4.read()\n",
    "print res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "jieba.suggest_freq('桓温', True)\n",
    "with open('./nlp_test4.txt') as f:\n",
    "    document3 = f.read()\n",
    "    \n",
    "    document3_decode = document3.decode('GBK')\n",
    "    document3_cut = jieba.cut(document3_decode)\n",
    "    #print  ' '.join(jieba_cut)\n",
    "    result = ' '.join(document3_cut)\n",
    "    result = result.encode('utf-8')\n",
    "    with open('./nlp_test5.txt', 'w') as f3:\n",
    "        f3.write(result)\n",
    "f.close()\n",
    "f3.close()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 44)\t0.154467434933\n",
      "  (0, 59)\t0.108549295069\n",
      "  (0, 39)\t0.308934869866\n",
      "  (0, 53)\t0.108549295069\n",
      "  (0, 65)\t0.108549295069\n",
      "  (0, 49)\t0.108549295069\n",
      "  (0, 40)\t0.108549295069\n",
      "  (0, 20)\t0.0772337174664\n",
      "  (0, 62)\t0.108549295069\n",
      "  (0, 10)\t0.108549295069\n",
      "  (0, 41)\t0.154467434933\n",
      "  (0, 56)\t0.108549295069\n",
      "  (0, 35)\t0.108549295069\n",
      "  (0, 24)\t0.108549295069\n",
      "  (0, 12)\t0.154467434933\n",
      "  (0, 2)\t0.217098590137\n",
      "  (0, 15)\t0.108549295069\n",
      "  (0, 17)\t0.108549295069\n",
      "  (0, 26)\t0.217098590137\n",
      "  (0, 0)\t0.217098590137\n",
      "  (0, 23)\t0.108549295069\n",
      "  (0, 57)\t0.108549295069\n",
      "  (0, 66)\t0.108549295069\n",
      "  (0, 64)\t0.108549295069\n",
      "  (0, 18)\t0.108549295069\n",
      "  :\t:\n",
      "  (1, 55)\t0.0995336411066\n",
      "  (1, 54)\t0.0995336411066\n",
      "  (1, 43)\t0.419673178975\n",
      "  (1, 37)\t0.139891059658\n",
      "  (1, 11)\t0.279782119316\n",
      "  (1, 16)\t0.279782119316\n",
      "  (1, 9)\t0.139891059658\n",
      "  (1, 8)\t0.139891059658\n",
      "  (1, 42)\t0.279782119316\n",
      "  (1, 14)\t0.139891059658\n",
      "  (1, 52)\t0.139891059658\n",
      "  (1, 28)\t0.139891059658\n",
      "  (1, 46)\t0.139891059658\n",
      "  (1, 33)\t0.139891059658\n",
      "  (1, 3)\t0.139891059658\n",
      "  (1, 6)\t0.139891059658\n",
      "  (1, 61)\t0.139891059658\n",
      "  (1, 36)\t0.279782119316\n",
      "  (1, 21)\t0.139891059658\n",
      "  (1, 29)\t0.139891059658\n",
      "  (1, 5)\t0.139891059658\n",
      "  (1, 27)\t0.139891059658\n",
      "  (1, 47)\t0.139891059658\n",
      "  (1, 30)\t0.139891059658\n",
      "  (1, 60)\t0.139891059658\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "corpus = [res1,res2]\n",
    "vector = TfidfVectorizer(stop_words=stpwrdlst)\n",
    "tfidf = vector.fit_transform(corpus)\n",
    "print tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------第 0 段文本的词语tf-idf权重------\n",
      "一起 0.217098590137\n",
      "万块 0.217098590137\n",
      "三人 0.217098590137\n",
      "三套 0.0\n",
      "下海经商 0.217098590137\n",
      "不想 0.0\n",
      "不要 0.0\n",
      "东挪西撮 0.108549295069\n",
      "之后 0.0\n",
      "事业有成 0.0\n",
      "事对 0.108549295069\n",
      "京州 0.0\n",
      "他们 0.154467434933\n",
      "以沫 0.108549295069\n",
      "公司 0.0\n",
      "分开 0.108549295069\n",
      "别墅 0.0\n",
      "前一晚 0.108549295069\n",
      "县当 0.108549295069\n",
      "县长 0.108549295069\n",
      "可是 0.0772337174664\n",
      "名下 0.0\n",
      "后来 0.108549295069\n",
      "喝酒 0.108549295069\n",
      "回忆起 0.108549295069\n",
      "困难 0.108549295069\n",
      "大家 0.217098590137\n",
      "太大 0.0\n",
      "她们 0.0\n",
      "好像 0.0\n",
      "家住 0.0\n",
      "容易 0.108549295069\n",
      "对不起 0.217098590137\n",
      "帝豪园 0.0\n",
      "开始 0.108549295069\n",
      "很大 0.108549295069\n",
      "房子 0.0\n",
      "打听 0.0\n",
      "时期 0.108549295069\n",
      "易学习 0.308934869866\n",
      "有福 0.108549295069\n",
      "李达康 0.154467434933\n",
      "欧阳 0.0\n",
      "毛娅 0.0\n",
      "沙瑞金 0.154467434933\n",
      "没想到 0.108549295069\n",
      "没有 0.0\n",
      "浪费 0.0\n",
      "王大路 0.386168587332\n",
      "百姓 0.108549295069\n",
      "相助 0.108549295069\n",
      "竟然 0.108549295069\n",
      "股权 0.0\n",
      "胸怀 0.108549295069\n",
      "自己 0.0772337174664\n",
      "觉得 0.154467434933\n",
      "触动 0.108549295069\n",
      "话别 0.108549295069\n",
      "赔礼道歉 0.108549295069\n",
      "赞叹 0.108549295069\n",
      "踏实 0.0\n",
      "这些 0.0\n",
      "这件 0.108549295069\n",
      "连连 0.108549295069\n",
      "道口 0.108549295069\n",
      "金山 0.108549295069\n",
      "降职 0.108549295069\n",
      "风生水 0.108549295069\n",
      "-------第 1 段文本的词语tf-idf权重------\n",
      "一起 0.0\n",
      "万块 0.0\n",
      "三人 0.0\n",
      "三套 0.139891059658\n",
      "下海经商 0.0\n",
      "不想 0.139891059658\n",
      "不要 0.139891059658\n",
      "东挪西撮 0.0\n",
      "之后 0.139891059658\n",
      "事业有成 0.139891059658\n",
      "事对 0.0\n",
      "京州 0.279782119316\n",
      "他们 0.0995336411066\n",
      "以沫 0.0\n",
      "公司 0.139891059658\n",
      "分开 0.0\n",
      "别墅 0.279782119316\n",
      "前一晚 0.0\n",
      "县当 0.0\n",
      "县长 0.0\n",
      "可是 0.0995336411066\n",
      "名下 0.139891059658\n",
      "后来 0.0\n",
      "喝酒 0.0\n",
      "回忆起 0.0\n",
      "困难 0.0\n",
      "大家 0.0\n",
      "太大 0.139891059658\n",
      "她们 0.139891059658\n",
      "好像 0.139891059658\n",
      "家住 0.139891059658\n",
      "容易 0.0\n",
      "对不起 0.0\n",
      "帝豪园 0.139891059658\n",
      "开始 0.0\n",
      "很大 0.0\n",
      "房子 0.279782119316\n",
      "打听 0.139891059658\n",
      "时期 0.0\n",
      "易学习 0.0995336411066\n",
      "有福 0.0\n",
      "李达康 0.0995336411066\n",
      "欧阳 0.279782119316\n",
      "毛娅 0.419673178975\n",
      "沙瑞金 0.0995336411066\n",
      "没想到 0.0\n",
      "没有 0.139891059658\n",
      "浪费 0.139891059658\n",
      "王大路 0.29860092332\n",
      "百姓 0.0\n",
      "相助 0.0\n",
      "竟然 0.0\n",
      "股权 0.139891059658\n",
      "胸怀 0.0\n",
      "自己 0.0995336411066\n",
      "觉得 0.0995336411066\n",
      "触动 0.0\n",
      "话别 0.0\n",
      "赔礼道歉 0.0\n",
      "赞叹 0.0\n",
      "踏实 0.139891059658\n",
      "这些 0.139891059658\n",
      "这件 0.0\n",
      "连连 0.0\n",
      "道口 0.0\n",
      "金山 0.0\n",
      "降职 0.0\n",
      "风生水 0.0\n"
     ]
    }
   ],
   "source": [
    "wordlist = vector.get_feature_names()#获取词袋模型中的所有词  \n",
    "# tf-idf矩阵 元素a[i][j]表示j词在i类文本中的tf-idf权重\n",
    "weightlist = tfidf.toarray()  \n",
    "#打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重\n",
    "for i in range(len(weightlist)):  \n",
    "    print \"-------第\",i,\"段文本的词语tf-idf权重------\"  \n",
    "    for j in range(len(wordlist)):  \n",
    "        print wordlist[j],weightlist[i][j]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
